##Testing code
library(tm)
require(parallel)
source("dumpdata.R")

df_sample<-read.csv("my_test.csv",stringsAsFactors=F)
perf.info<-read.csv("perf_info.csv",stringsAsFactors=F)

dtm.text<-tolower(paste(df_sample$Title, df_sample$Body, sep=" ")) ##combine title and body and convert to lower case
text.corpus<-Corpus(VectorSource(dtm.text))

##Now clear out all stop words
mystops<-c(
 "a","about","above","after","again","against","all","am",
"an","and","any","appreciate","aren't","are","as","at",
"be","because","been","before","being","below","between",
"both","but","by","can't","cannot","couldn't","could",
"didn't","did","do","doesn't","does","doing","don't",
"down","during","each","few","for","from","further",
"hadn't","had","hasn't","has","haven't","have","having",
"he'd","he'll","he's","he","help","her","here's","here",
"hers","herself","him","himself","his","how's","how",
"i'd","i'll","i'm","i've","i","if","in","into","isn't",
"is","it's","it","its","itself","let's","me","more",
"most","mustn't","my","myself","no","nor","not","of",
"off","on","once","only","or","other","ought","our",
"ours ","ourselves","out","over","own","same","shan't",
"she'd","she'll","she's","shouldn't","should","she","so",
"some","such","than","thank","thanks","that's","that","the",
"theirs","their","them","themselves","then","there's","there",
"these","they'd","they'll","they're","they've","they",
"this","those","through","to","too","under","until","up",
"very","wasn't","was","we'd","we'll","we're","we've",
"we","were","weren't","what's","what","when's","when",
"where's","where","which","while","who's","who","whom",
"why's","why","with","won't","wouldn't","would","you'd",
"you'll","you're","you've","you","your","yours","yourself",
"your","application", "can", "code", "create", "data", "error",
"find", "following", "get", "just", "know","like", "need", "pre",
"use", "using", "want", "way", "will","now", "one", "problem",
 "something", "sure", "trying", "work", "right", "run", "running",
 "see", "seems", "set", "show", "similar", "simple", "since",
 "possible",  "value","method","also","app","time","works" , "please"
)

text.corpus<-tm_map(text.corpus, removeWords, mystops )
text.corpus<-tm_map(text.corpus, removeWords, stopwords("english"))

dtm.text<-unlist(text.corpus)

dtm.text<- gsub("<code>|</code>|<p>|</p>|<strong>|</strong>|<pre>|</pre>"," ",dtm.text, perl=T)
dtm.text<- gsub("\n"," ",dtm.text, fixed=T)
dtm.text<- gsub("c#","csharp",dtm.text, fixed=T)               ##So c# and c++ will be recognized as term
dtm.text<- gsub("f#","fsharp",dtm.text, fixed=T)
dtm.text<- gsub("c++","cplusplus",dtm.text, fixed=T)
dtm.text<-gsub("[[:punct:]]", " ", dtm.text, perl=T)

text.corpus<-Corpus(VectorSource(dtm.text))
text.corpus<-tm_map(text.corpus, stripWhitespace)
dtm <- removeSparseTerms(DocumentTermMatrix(text.corpus), 0.999)
dtm2<-as.matrix(dtm)
remove(dtm)
gc()


v.text<-tolower(paste(df_sample$Title, df_sample$Body, sep=" "))
v.title<-tolower(df_sample$Title)



f<-function(i)
{
  print(i)
  re2<-perf.info$tag.map.regexp[i]
  if(b[[i]]$pred.type=="grep")
    ind.pred<-grep(re2, v.text, perl=T)

  if(b[[i]]$pred.type=="model")
    {
      coeffs<-b[[i]]$model.coeff
      new.names<-setdiff(names(coeffs)[-1], colnames(dtm2))
      old.names<-intersect(names(coeffs)[-1], colnames(dtm2))
      new.x<-matrix(0, length(v.text), length(new.names))
      colnames(new.x)<-new.names
      if(length(old.names)==1) {old.x<-matrix(dtm2[,old.names], length(v.text), 1); colnames(old.x)<-old.names} else old.x<-dtm2[,old.names]
      X<-cbind("(Intercept)"=1 , old.x, new.x)
      X<-X[,names(coeffs)]
      probs<-plogis(X%*%coeffs)
      predTF<-probs>b[[i]]$threshold
      ind.pred<-(1:length(v.text))[predTF]
    }

  if(b[[i]]$pred.type=="grep title")
      ind.pred<-grep(re2, v.title, perl=T)

  ind.pred
}

ans<-mclapply(1:length(b), f, mc.cores=16)

predictions<-rep("",length(v.text))
for(i in 1:length(ans))
{
  ind<-ans[[i]]
  predictions[ind]<-paste(predictions[ind],perf.info$tag[i])
}

predictions<-substring(predictions, 2)

num.to.keep<-function(p)
{
  a<-vector("list",length(p))
  a<-lapply(1:length(a), function(i) a[[i]]<-c(0,1))
  outcomes<-expand.grid(a)
  prob.outcome<-sapply(1:nrow(outcomes),function(i)prod(p[unlist(outcomes[i,]>0)])*prod(1-p[unlist(outcomes[i,]==0)]))
  Ef<-rep(NA, length(p))
  for(j in 1:length(p))
  {
    a.try<-rep(0, length(p))
    a.try[1:j]<-1
    tp<-as.matrix(outcomes)%*%a.try
    prec<-tp/j
    rec<-tp/apply(outcomes, 1, sum)
    f<-2*prec*rec/(prec+rec)
    f[is.na(f)]<-0
    Ef[j]<-sum(f*prob.outcome)
    #cat(j, Ef, "\n")
  }
  if(max(Ef)<.1) 0
  else which.max(Ef)
}

##Now predictions must be ordered by specificity and only the most powerful ones retained
for(i in 1:length(predictions)){
  print(i)
  if(predictions[i]=="") next
  v_pred<-unlist(strsplit(predictions[i]," "))
  v_spec<-sapply(1:length(v_pred),function(j) perf.info$spec[perf.info$tag==v_pred[j]])
  v_pred<-v_pred[rev(order(v_spec))]
  v_spec<-v_spec[rev(order(v_spec))]
  if (length(v_pred)>5)
  {
    v_pred<-v_pred[1:5]
    v_spec<-v_spec[1:5]
  }
  n<-num.to.keep(v_spec)
  cat(length(v_pred), " predictions, keeping ", n, "\n")
  if (n==0)
    predictions[i]<-""
  else
    predictions[i]<-paste(v_pred[1:n],collapse=" ")
}

predictions[predictions==""]<-"c# java javascript c++"

F1<-function(pred,actual)
{
	tp<-sum(sapply(1:length(pred), function(n) max(actual==pred[n])))
	if(tp==0) return(0)
	prec<-tp/length(pred)
	recall<-tp/length(actual)
	2*prec*recall/(prec+recall)
}


mean(sapply(1:length(predictions), function (n) F1(unlist(strsplit(predictions[n],split=" ")),
                                                  unlist(strsplit(df_sample$Tags[n], split=" ")))))

















##Now predictions must be ordered by specificity and only the most powerful ones retained
for(i in 1:length(test.predictions)){
print(i)
if(test.predictions[i] == "") next
v_pred<-unlist(strsplit(test.predictions[i], " "))
v_spec<-sapply(1:length(v_pred), function(j) model.info$spec[model.info$tag==v_pred[j]])
v_pred<-v_pred[rev(order(v_spec))]
v_spec<-v_spec[rev(order(v_spec))]
v_spec[is.na(v_spec)]<-0
if(max(v_spec)<=.15) test.predictions[i]<-""
else test.predictions[i]<-paste(v_pred[v_spec>.15],collapse=" ")
}

test.predictions[test.predictions==""]<-"java c#"


F1<-function(pred,actual)
{
	tp<-sum(sapply(1:length(pred), function(n) max(actual==pred[n])))
	if(tp==0) return(0)
	prec<-tp/length(pred)
	recall<-tp/length(actual)
	2*prec*recall/(prec+recall)
}

mean(sapply(1:length(test.predictions), function (n) F1(unlist(strsplit(test.predictions[n],split=" ")),
                                                  unlist(strsplit(df_test$Tags[n], split=" ")))))
